package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSONObject;
import com.google.gson.GsonBuilder;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.api.realize.AbstractUserCrawler;
import com.spider.moudle.entity.*;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.SettingsUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;

/**
 * 编码：gb2312
 * Created by qiuhao on 2017/12/27.
 */
public class TieXueCrawler extends AbstractCrawler {

    protected Downloader downloader = DownloaderFactory.buildDownloader("gb18030");
    /**
     * 这个是每个网站的个性化配置，是从settings.json里面读取到的(必须)
     */
    private Setting setting = SettingsUtil.getSettingBySite("TieXue");


    private int friendPageCount = 10;  //最多爬取10页好友信息


    private String index = setting.getIndex();

    private String defaultUrl = "http://bbs.tiexue.net";


    @Override
    public void generateTask() {
        List<String> list = new ArrayList<>();

        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                entity = downloader.get(index, true);

            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                Document document = entity.getDocument();
                Elements elements = document.select(".list_tit");
                if (elements.isEmpty()) {
                    return;
                }
                for (int i = 2; i < 19; i++) {
                    if (i == 2) {
                        String url = elements.get(i).select("a").get(0).attr("href");
                        list.add(url);
                        continue;
                    }
                    Elements aElements = elements.get(i).select("ul").get(0).select("a");
                    if (aElements != null && aElements.size() > 0) {
                        for (int j = 0; j < aElements.size(); j++) {
                            String url = aElements.get(j).attr("href");
                            list.add(defaultUrl + url);
                        }
                    }
                }
                executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()),
                        new NamedThreadFactory(setting.getName(), "生成任务"));
                for (int i = 0; i < list.size(); i++) {
                    TieXueCrawler.TieXueTaskCrawler default_task = new TieXueCrawler.TieXueTaskCrawler(list.get(i));
                    executorService.submit(default_task);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != executorService) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    /**
     * 爬取帖子内容和用户主页
     *
     * @param task
     * @return
     * @throws Exception
     */
    @Override
    public JSONObject crawlItem(Task task) throws Exception {

        //帖子
        if (task.getTaskType() == TaskType.SPEECH) {
            SpeechBean speechBean = new SpeechBean();
            int read_count = Integer.valueOf(task.get("read").toString());
            String title = task.get("title").toString();
            speechBean.setRead_count(read_count);
            speechBean.setTitle(title);
            JSONObject item = new JSONObject();
            String url = task.getUrl();
            speechBean.setUrl(url);
            speechBean.setArea(task.getArea());
            ResponseEntity entity = null;
            String publlish_user = "";
            String publish_user_id = "";
            String publish_user_photo = "http://r.itiexue.net/tiexue/usercenter/usericon.gif";
            String content = "";
            String id = task.get("id").toString();
            try {
                entity = downloader.get(url, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            int page = 1;
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
                    Elements postElements = document.select(".postStart");
                    if (postElements != null && postElements.size() > 0) {
                        Element speechContentElement = postElements.get(0);   //主贴和发帖人信息
                        Elements headElements = speechContentElement.select(".headPic"); //头像
                        if (headElements != null && headElements.size() > 0) {
                            publish_user_photo = headElements.get(0).select("img").get(0).absUrl("src");

                        }
                        Elements userInfoElements = speechContentElement.select(".userName").get(0).select("a");
                        publlish_user = userInfoElements.get(0).text();
                        publish_user_id = userInfoElements.get(0).attr("href").split("/")[3];
                        String user_url = userInfoElements.get(0).absUrl("href");
                        //将发帖人添加到任务
                        if (CommonUtil.isNotNull(user_url)) {
                            String userKey = setting.getName() + "_" + publish_user_id;
                            Task hometask = new Task(user_url + "/index.html", TaskType.USER, WebType.FORLUM, userKey);
                            hometask.put("publish_user", publlish_user);
                            hometask.put("publish_user_photo", publish_user_photo);
                            TaskManager.add(setting.getUser_task(), hometask);
                        }
                        List<String> img_urls = new ArrayList<>();
                        Elements contentElements = speechContentElement.select("#postContent"); //内容
                        if (contentElements != null && contentElements.size() > 0) {
                            content = contentElements.get(0).text();

                            Elements imgElements = contentElements.get(0).select("img");
                            if (imgElements != null && imgElements.size() > 0) {
                                for (int i = 0; i < imgElements.size(); i++) {
                                    img_urls.add(imgElements.get(i).absUrl("src"));
                                }
                            }

                        }

                        String publish_time = CommonUtil.trim(speechContentElement.select(".date").text());
                        speechBean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, publish_time.replaceAll("/", "-")));
                        Date now = new Date();
                        speechBean.setSpider_time(now);
                        speechBean.setPublish_user_jsonid(setting.getName() + "_" + publish_user_id);
                        speechBean.setPublish_user(publlish_user);
                        speechBean.setPublish_user_photo(publish_user_photo);
                        speechBean.setPublish_user_id(publish_user_id);
                        speechBean.setContent(content.trim());
                        speechBean.setImg_urls(img_urls);
                        speechBean.setId(id);
                        speechBean.setUrl(url);
                        List<SpeechBean> reply_nodes = new ArrayList<>();
                        //第一页的回复
                        Elements replyEntity = null;
                        Elements nameElements = null;
                        try {
                            replyEntity = document.select("#PostReply_List").get(0).select(".postStart");
                            nameElements = document.select("#PostReply_List").get(0).select("a[name]");
                        } catch (Exception e) {
                            System.out.println("获取第一页回复出错：" + url);
                            e.printStackTrace();
                        }
                        if (replyEntity != null && replyEntity.size() > 0) {
                            for (int i = 0; i < replyEntity.size(); i++) {
                                SpeechBean reply = new SpeechBean();
                                String reply_id = nameElements.get(i).attr("name");
                                Element userMsg = replyEntity.get(i).select(".userMsg").get(0);
                                String reply_publish_user = userMsg.select("a").get(0).text();
                                String reply_publish_user_url = userMsg.select("a").get(0).absUrl("href");
                                String reply_publish_user_id = userMsg.select("a").get(0).attr("href").split("/")[3];
                                String reply_publish_user_photo = "http://r.itiexue.net/tiexue/usercenter/usericon.gif";
                                Elements picElements = userMsg.select("a").get(0).select(".headPic");
                                if (picElements != null && picElements.size() > 0) {
                                    reply_publish_user_photo = picElements.get(0).select("img").get(0).attr("src");
                                }

                                if (CommonUtil.isNotNull(reply_publish_user_url)) {
                                    String userKey = setting.getName() + "_" + reply_publish_user_id;
                                    Task hometask = new Task(reply_publish_user_url + "/index.html", TaskType.USER, WebType.FORLUM, userKey);
                                    hometask.put("publish_user", reply_publish_user);
                                    hometask.put("publish_user_photo", reply_publish_user_photo);
                                    TaskManager.add(setting.getUser_task(), hometask);
                                }

                                String reply_publish_time = CommonUtil.trim(replyEntity.get(i).select(".date").get(0).text());

                                Element contentElement = replyEntity.get(i).select(".bbsp2").get(0);
                                String parentId = task.get("id").toString();
                                Elements yinyongElements = contentElement.select(".yinyong");
                                if (yinyongElements != null && yinyongElements.size() > 0) {
                                    Elements dtElements = yinyongElements.get(0).select("dt");
                                    if (dtElements != null && dtElements.size() > 0) {
                                        dtElements.get(dtElements.size() - 1).select("a").remove();
                                        parentId = CommonUtil.trim(dtElements.get(dtElements.size() - 1).text()).split("楼")[0];
                                    }
                                }
                                yinyongElements.remove();

                                Elements quoteTableElements = contentElement.select(".quoteTable");
                                if (quoteTableElements != null && quoteTableElements.size() > 0) {
                                    Elements dtElements = quoteTableElements.get(0).select("strong");
                                    if (dtElements != null && dtElements.size() > 0) {
//                                        dtElements.get(0).select("a").remove();
                                        parentId = CommonUtil.trim(dtElements.get(0).text()).split("楼")[0].split("第")[1];
                                    }
                                }
                                quoteTableElements.remove();

                                String reply_content = contentElement.text();
                                List<String> reply_img_urls = new ArrayList<>();
                                Elements replyImg = contentElement.select("img");
                                if (replyImg != null && replyImg.size() > 0) {
                                    for (int j = 0; j < replyImg.size(); j++) {
                                        String img = replyImg.get(j).attr("src");
                                        if (!img.contains("http:")) {
                                            img = "http://bbs.tiexue.net/" + img;
                                        }
                                        reply_img_urls.add(img);
                                    }
                                }
                                reply.setPublish_user(reply_publish_user);
                                reply.setPublish_user_id(reply_publish_user_id);
                                reply.setPublish_user_jsonid(setting.getName() + "_" + reply_publish_user_id);
                                reply.setPublish_user_photo(reply_publish_user_photo);
                                reply.setId(reply_id);
                                reply.setContent(reply_content.trim());
                                reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_publish_time.replaceAll("/", "-")));
                                reply.setUrl(url + "#" + reply_id);
                                reply.setParent_id(parentId);
                                reply.setAncestor_id(id);
                                reply.setImg_urls(reply_img_urls);

                                reply_nodes.add(reply);
                            }

                            Elements pageElements = document.select(".page");
                            if (pageElements != null && pageElements.size() > 0) {
                                int p = Integer.valueOf(pageElements.get(1).select("span").get(0).text());
                                if (p != 1) {
                                    page = p + 1;
                                }
                            }

                            if (page > 1) {
                                //爬虫最大页码的限制，超过最大页的帖子就只抓到最大页码数
                                if (page >= setting.getMax_page_count()) {
                                    page = setting.getMax_page_count();
                                }
                                int threadNum = setting.getReply_thread_count();
                                List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                                ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, page, threadNum);
                                ExecutorService executorService = null;
                                try {
                                    executorService = ExecutorServiceFactoy.buildExecutorService(threadNum, ExecutorServiceFactoy.getName(setting.getName(), url), new NamedThreadFactory(setting.getName(), "回复"));
                                    for (int m = 0; m < splitPages.size(); m++) {
                                        ReplyCrawler callable = new ReplyCrawler(splitPages.get(m), url, id, id);
                                        FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                    log.error("天涯社区获取回复线程执行异常");
                                } finally {
                                    if (null != executorService) {
                                        executorService.shutdown();
                                    }
                                }

                                for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                                    reply_nodes.addAll(futureTask.get());
                                }
                                Collections.sort(reply_nodes, (o1, o2) -> {
                                    Date s1 = o1.getPublish_time();
                                    Date s2 = o2.getPublish_time();
                                    int flag = s1.compareTo(s2);
                                    return flag;
                                });

                            }
                            speechBean.setReply_nodes(reply_nodes);
                        }
                        speechBean.setReply_count(reply_nodes.size());
                        item.put("data", speechBean.toJSONObject());
                        return item;
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }

            }


            //用户主页
        } else if (task.getTaskType() == TaskType.USER) {
            String userKey = task.getTaskKey();
            if (CommonUtil.isNull(userKey)) {
                System.err.println("User key is not empty where task type belong to home page");
                return null;
            }
            JSONObject item = new JSONObject();
            UserBean data;
            String url = task.getUrl();
            String detailUrl = url.replaceAll("/index.html", "") + "?nexturl=" + url;
            String publish_user_photo = "";
            ResponseEntity photoentity = null;
            try {
                photoentity = downloader.get(detailUrl, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (photoentity != null && photoentity.getStatus_code() == 200) {
                Document document = photoentity.getDocument();
                Element srcElement = null;
                try {
                    srcElement = document.select("#userhead").get(0);
                    String src = srcElement.attr("src");
                    if (!src.contains("http")) {
                        publish_user_photo = (String) task.get("publish_user_photo");
                    } else {
                        publish_user_photo = src;
                    }
                } catch (Exception e) {
                    System.out.println("获取用户的默认头像出错：" + detailUrl);
                    publish_user_photo = (String) task.get("publish_user_photo");
                }

            }

            ResponseEntity entity = null;
            try {
                entity = downloader.get(url, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
//                    Element iframepage=document.select("#iframepage").get(0);

                    data = new UserBean();
                    data.setJson_id(userKey);
                    data.setUrl(url);
                    String publish_user_id = userKey.substring(userKey.indexOf("_") + 1);
                    data.setPublish_user_id(publish_user_id);
                    String publish_user = (String) task.get("publish_user");
                    data.setPublish_user(publish_user);
                    data.setPublish_user_photo(publish_user_photo);
                    int article_count = 0;
                    int reply_article_count = 0;
                    int friend_count = 0;
                    String register_time = "";
                    //好友(有好友就需要爬好友信息)
                    List<UserBean> friend_details = new ArrayList<>();
                    //粉丝(有粉丝就需要爬粉丝信息)
                    List<UserBean> fans_details = new ArrayList<>();
                    //关注(有关注用户就需要爬关注用户信息)
                    List<UserBean> follow_details = new ArrayList<>();
                    Elements basicInfo = document.select(".infoBox");
                    if (basicInfo != null && basicInfo.size() > 0) {
                        Elements infoElements = basicInfo.get(0).select(".xbinfo");
                        if (!infoElements.isEmpty()) {
                            String sex = infoElements.get(0).select("em").get(0).text();
                            article_count = Integer.valueOf(infoElements.get(0).select("em").get(2).text().split("篇")[0]);
                            reply_article_count = Integer.valueOf(infoElements.get(0).select("em").get(4).text().split("篇")[0]);
                            friend_count = Integer.valueOf(infoElements.get(0).select("em").get(5).text().split("位")[0]);
                            data.setSex(sex);

                        }
                        Elements timeElements = basicInfo.get(0).select(".times");
                        if (timeElements != null && timeElements.size() > 0) {
                            register_time = timeElements.get(0).select("em").get(0).text().replaceAll("\\.", "-");
                        }
                    }
                    if (friend_count > 0) {
                        String defaultPhoto = "http://r.itiexue.net/tiexue/usercenter/usericon.gif";
                        Elements followElements = document.select(".hyTitle");
                        if (followElements != null && followElements.size() > 0) {
                            Element followElement = followElements.get(0);
                            String followPageurl = followElement.select("a").get(0).attr("href");
                            ResponseEntity friendEntity = null;
                            try {
                                friendEntity = downloader.get(url.replaceAll("index.html", "") + followPageurl, true);
                            } catch (Exception e) {
                                log.error(e.getMessage(), e);
                            }
                            if (friendEntity != null && friendEntity.getStatus_code() == 200) {
                                Document friendDoc = friendEntity.getDocument();
                                Elements items = friendDoc.select(".item");
                                if (items != null && items.size() > 0) {
                                    for (int i = 0; i < items.size(); i++) {
                                        UserBean followBean = new UserBean();
                                        Element friendInfo = items.get(i).select(".float_L.head").get(0);
                                        String friendUrl = friendInfo.select("a").get(0).attr("href");
                                        String friendId = friendUrl.split("/")[3];
                                        String friendName = friendInfo.select("a").get(0).attr("title");
                                        String friendPhoto = friendInfo.select("a").get(0).select("img").get(0).attr("src");
                                        String friendUserKey = setting.getName() + "_" + friendId;
                                        if (friendPhoto.equals("")) {
                                            friendPhoto = defaultPhoto;
                                        }
                                        followBean.setPublish_user(friendName);
                                        followBean.setPublish_user_photo(friendPhoto);
                                        followBean.setPublish_user_id(friendId);
                                        followBean.setUrl(friendUrl);
                                        followBean.setJson_id(friendUserKey);
                                        follow_details.add(followBean);
                                    }
                                }

                                Elements pageElements = friendDoc.select(".page");
                                if (pageElements != null && pageElements.size() > 0) {
                                    Elements aElements = pageElements.get(0).select("a");
                                    if (aElements.size() > 0) {
                                        String params[] = aElements.get(1).attr("href").split("_");
                                        int pagenum = Integer.valueOf(params[1].split("\\.")[0]);
                                        if (pagenum > friendPageCount) {
                                            pagenum = friendPageCount;
                                        }
                                        int threadNum = 1;
                                        if (pagenum > 4) {
                                            threadNum = pagenum / 2;
                                        }
                                        ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, pagenum, threadNum);
                                        List<FutureTask<List<UserBean>>> futureTasks = new ArrayList<>();
                                        ExecutorService executorService = null;
                                        try {
                                            executorService = ExecutorServiceFactoy.buildExecutorService(splitPages.size(), ExecutorServiceFactoy.getName(setting.getName(), followPageurl),
                                                    new NamedThreadFactory(setting.getName(), "用户好友"));
                                            for (int i = 0; i < splitPages.size(); i++) {
                                                TieXueCrawler.FriendUserCramler callable = new TieXueCrawler.FriendUserCramler(splitPages.get(i), url.replaceAll("index.html", "") + followPageurl, setting.getName());
                                                FutureTask<List<UserBean>> futureTask = new FutureTask<>(callable);
                                                futureTasks.add(futureTask);
                                                executorService.submit(futureTask);
                                            }
                                            if (futureTasks.size() > 0) {
                                                for (FutureTask<List<UserBean>> futureTask : futureTasks) {
                                                    follow_details.addAll(futureTask.get());
                                                }
                                            }
                                        } catch (Exception e) {
                                            e.printStackTrace();
                                        }
                                    }
                                }
                            }


                            //获取粉丝
                            ResponseEntity fansEntity = null;
                            String fansUrl = "fans.html";
                            try {
                                fansEntity = downloader.get(url.replaceAll("index.html", "") + fansUrl, true);
                            } catch (Exception e) {
                                log.error(e.getMessage(), e);
                            }
                            if (fansEntity != null && fansEntity.getStatus_code() == 200) {
                                Document fansDoc = fansEntity.getDocument();
                                Elements fansitems = fansDoc.select(".item");
                                if (fansitems != null && fansitems.size() > 0) {
                                    for (int i = 0; i < fansitems.size(); i++) {
                                        UserBean fansBean = new UserBean();
                                        Element fansInfo = fansitems.get(i).select(".float_L.head").get(0);
                                        String fansurl = fansInfo.select("a").get(0).attr("href");
                                        String fansId = fansurl.split("/")[3];
                                        String fansName = fansInfo.select("a").get(0).attr("title");
                                        String fansPhoto = fansInfo.select("a").get(0).select("img").get(0).attr("src");
                                        String fansUserKey = setting.getName() + "_" + fansId;
                                        if (fansPhoto.equals("")) {
                                            fansPhoto = defaultPhoto;
                                        }
                                        fansBean.setPublish_user(fansName);
                                        fansBean.setPublish_user_photo(fansPhoto);
                                        fansBean.setPublish_user_id(fansId);
                                        fansBean.setUrl(fansurl);
                                        fansBean.setJson_id(fansUserKey);
                                        fans_details.add(fansBean);
                                    }
                                }
                                Elements pageElements = fansDoc.select(".page");
                                if (pageElements != null && pageElements.size() > 0) {
                                    Elements aElements = pageElements.get(0).select("a");
                                    if (aElements.size() > 0) {
                                        String params[] = aElements.get(1).attr("href").split("_");
                                        int pagenum = Integer.valueOf(params[1].split("\\.")[0]);
                                        if (pagenum > friendPageCount) {
                                            pagenum = friendPageCount;
                                        }
                                        int threadNum = 1;
                                        if (pagenum > 4) {
                                            threadNum = pagenum / 2;
                                        }
                                        ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, pagenum, threadNum);
                                        List<FutureTask<List<UserBean>>> futureTasks = new ArrayList<>();
                                        ExecutorService executorService = null;
                                        try {
                                            executorService = ExecutorServiceFactoy.buildExecutorService(splitPages.size(), ExecutorServiceFactoy.getName(setting.getName(), fansUrl),
                                                    new NamedThreadFactory(setting.getName(), "用户好友"));
                                            for (int i = 0; i < splitPages.size(); i++) {
                                                TieXueCrawler.FriendUserCramler callable = new TieXueCrawler.FriendUserCramler(splitPages.get(i), url.replaceAll("index.html", "") + fansUrl, setting.getName());
                                                FutureTask<List<UserBean>> futureTask = new FutureTask<>(callable);
                                                futureTasks.add(futureTask);
                                                executorService.submit(futureTask);
                                            }
                                            if (futureTasks.size() > 0) {
                                                for (FutureTask<List<UserBean>> futureTask : futureTasks) {
                                                    fans_details.addAll(futureTask.get());
                                                }
                                            }
                                        } catch (Exception e) {
                                            e.printStackTrace();
                                        }
                                    }
                                }
                            }

                            if (fans_details.size() > 0 && follow_details.size() > 0) {
                                for (int i = 0; i < fans_details.size(); i++) {
                                    for (int j = 0; j < follow_details.size(); j++) {
                                        if (fans_details.get(i).getPublish_user_id().equals(follow_details.get(j).getPublish_user_id())) {
                                            friend_details.add(fans_details.get(i));
                                            break;
                                        }
                                    }
                                }
                            }
                        }
                    }
                    data.setArticle_count(article_count);
                    data.setReply_article_count(reply_article_count);
                    try {
                        if(register_time.equals("")){
                            return null;
                        }
                        data.setRegister_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, register_time));
                    } catch (Exception e) {
                        System.out.println("获取注册时间出错:--->" + url);
                        e.printStackTrace();
                    }
                    data.setFriend_details(friend_details);
                    data.setFriend_count(friend_count);
                    data.setFans_details(fans_details);
                    data.setFans_count(fans_details.size());
                    data.setFollow_details(follow_details);
                    data.setFollow_count(follow_details.size());
                    item.put("data", data.toJSONObject());
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }

            return item;
        }


        return null;
    }

    /**
     * 爬取好友
     */
    public class FriendUserCramler extends AbstractUserCrawler {

        private ArrayList<Integer> pages;

        private String settingName;

        private String publish_user_photo = "http://r.itiexue.net/tiexue/usercenter/usericon.gif";

        public FriendUserCramler(ArrayList<Integer> pages, String url, String settingName) {
            this.pages = pages;
            this.url = url;
            this.settingName = settingName;

        }

        @Override
        public List<UserBean> call() throws Exception {

            List<UserBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                String nextUrl = url.substring(0, url.length() - 5) + "_" + pages.get(k) + ".html";

                ResponseEntity friendEntity = null;
                try {
                    friendEntity = downloader.get(nextUrl, true);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
                if (friendEntity != null && friendEntity.getStatus_code() == 200) {
                    Document friendDoc = friendEntity.getDocument();

                    Elements items = friendDoc.select(".item");
                    if (items != null && items.size() > 0) {
                        for (int i = 0; i < items.size(); i++) {
                            UserBean followBean = new UserBean();
                            Element friendInfo = items.get(i).select(".float_L.head").get(0);
                            String friendUrl = friendInfo.select("a").get(0).attr("href");
                            String friendId = friendUrl.split("/")[3];
                            String friendName = friendInfo.select("a").get(0).attr("title");
                            String friendPhoto = friendInfo.select("a").get(0).select("img").get(0).attr("src");
                            String userKey = settingName + "_" + friendId;
                            if (friendPhoto.equals("")) {
                                friendPhoto = publish_user_photo;
                            }
                            followBean.setPublish_user(friendName);
                            followBean.setPublish_user_photo(friendPhoto);
                            followBean.setPublish_user_id(friendId);
                            followBean.setUrl(friendUrl);
                            followBean.setJson_id(userKey);
                            array.add(followBean);
                        }
                    }
                }

            }
            return array;
        }
    }


    /**
     * 抓回复的线程(这个是个性化的每个网站都不一样)
     */
    public class ReplyCrawler extends AbstractReplyCrawler {

        private ArrayList<Integer> pages;

        public ReplyCrawler(ArrayList<Integer> pages, String url, String parentId, String ancestorId) {
            this.pages = pages;
            this.url = url;
            this.parentId = parentId;
            this.ancestorId = ancestorId;

        }

        @Override
        public List<SpeechBean> call() throws Exception {
            List<SpeechBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                try {
                    String[] params = url.split("_");
                    String nextUrl = params[0] + "_" + params[1] + "_" + pages.get(k) + ".html";
                    Document document = null;
                    ResponseEntity entity = downloader.get(nextUrl, true);
                    if (entity.getStatus_code() == 200) {
                        document = entity.getDocument();
                        Elements replyEntity = document.select("#PostReply_List").get(0).select(".postStart");
                        Elements nameElements = document.select("#PostReply_List").get(0).select("a[name]");

                        if (replyEntity != null && replyEntity.size() > 0) {
                            for (int i = 0; i < replyEntity.size(); i++) {
                                SpeechBean reply = new SpeechBean();
                                String reply_id = nameElements.get(i).attr("name");
                                Element userMsg = replyEntity.get(i).select(".userMsg").get(0);
                                String reply_publish_user = userMsg.select("a").get(0).text();
                                String reply_publish_user_url = userMsg.select("a").get(0).absUrl("href");
                                String reply_publish_user_id = userMsg.select("a").get(0).attr("href").split("/")[3];
                                String reply_publish_user_photo = "http://r.itiexue.net/tiexue/usercenter/usericon.gif";
                                Elements picElements = userMsg.select("a").get(0).select(".headPic");
                                if (picElements != null && picElements.size() > 0) {
                                    reply_publish_user_photo = picElements.get(0).select("img").get(0).attr("src");
                                }

                                if (CommonUtil.isNotNull(reply_publish_user_url)) {
                                    String userKey = setting.getName() + "_" + reply_publish_user_id;
                                    Task hometask = new Task(reply_publish_user_url + "/index.html", TaskType.USER, WebType.FORLUM, userKey);
                                    hometask.put("publish_user", reply_publish_user);
                                    hometask.put("publish_user_photo", reply_publish_user_photo);
                                    TaskManager.add(setting.getUser_task(), hometask);
                                }

                                String reply_publish_time = CommonUtil.trim(replyEntity.get(i).select(".date").get(0).text());

                                Element contentElement = replyEntity.get(i).select(".bbsp2").get(0);
                                Elements yinyongElements = contentElement.select(".yinyong");
                                if (yinyongElements != null && yinyongElements.size() > 0) {
                                    Elements dtElements = yinyongElements.get(0).select("dt");
                                    dtElements.get(dtElements.size() - 1).select("a").remove();
                                    parentId = CommonUtil.trim(dtElements.get(dtElements.size() - 1).text()).split("楼")[0];
                                }
                                yinyongElements.remove();

                                Elements quoteTableElements = contentElement.select(".quoteTable");
                                if (quoteTableElements != null && quoteTableElements.size() > 0) {
                                    Elements dtElements = quoteTableElements.get(0).select("strong");
                                    if (dtElements != null && dtElements.size() > 0) {
//                                        dtElements.get(0).select("a").remove();
                                        parentId = CommonUtil.trim(dtElements.get(0).text()).split("楼")[0].split("第")[1];
                                    }
                                }
                                quoteTableElements.remove();

                                String reply_content = contentElement.text();
                                List<String> reply_img_urls = new ArrayList<>();
                                Elements replyImg = contentElement.select("img");
                                if (replyImg != null && replyImg.size() > 0) {
                                    for (int j = 0; j < replyImg.size(); j++) {
                                        reply_img_urls.add(replyImg.get(j).absUrl("src"));
                                    }
                                }
                                reply.setPublish_user(reply_publish_user);
                                reply.setPublish_user_id(reply_publish_user_id);
                                reply.setPublish_user_jsonid(setting.getName() + "_" + reply_publish_user_id);
                                reply.setPublish_user_photo(reply_publish_user_photo);
                                reply.setId(reply_id);
                                reply.setContent(reply_content.trim());
                                reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_publish_time.replaceAll("/", "-")));
                                reply.setUrl(url + "#" + reply_id);
                                reply.setParent_id(parentId);
                                reply.setAncestor_id(ancestorId);
                                reply.setImg_urls(reply_img_urls);
                                array.add(reply);
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            return array;
        }
    }


    /**
     * 生成任务
     */
    public class TieXueTaskCrawler implements Runnable {

        private String url;


        private Downloader downloader = DownloaderFactory.buildDownloader("gb18030");


        public TieXueTaskCrawler(String url) {
            this.url = url;

        }

        @Override
        public void run() {
            handle();
        }

        private void handle() {
            for (int p = 0; p < 2; p++) {
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(url, true);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements contentElements = document.select(".contens");
                        if (contentElements != null && contentElements.size() > 0) {
                            for (int i = 0; i < contentElements.size(); i++) {
                                Elements listTitleElements = contentElements.get(i).select(".listTitle");
                                if (listTitleElements != null && listTitleElements.size() > 0) {

                                    Elements cel04Elements = contentElements.get(i).select(".cel_04");
                                    int read = 0;
                                    if (cel04Elements != null && cel04Elements.size() > 0) {
                                        String readAndReply = cel04Elements.get(0).text();
                                        if (readAndReply.contains("/")) {
                                            read = Integer.valueOf(readAndReply.split("/")[0].trim());
                                        } else {
                                            read = Integer.valueOf(readAndReply.trim());
                                        }
                                    }
                                    String speechUrl = listTitleElements.get(0).attr("href");
                                    String speech_id = speechUrl.split("_")[1].trim();
                                    String speechkey = setting.getSite() + "_" + speech_id;
                                    String title = listTitleElements.get(0).text();
                                    Task task = new Task(speechUrl, TaskType.SPEECH, WebType.FORLUM, speechkey, null);
                                    task.put("read", read);
                                    task.put("title", title);
                                    task.put("id", speech_id);
                                    TaskManager.add(setting.getSpeech_task(), task);
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
}
